#include "parse.h"
#include "string.h"
#include "log.h"
#include "download_manager.h"
#include "url.h"
#include "page.h"
#include "util.h"
#include "dict_manager.h"

#include <vector>
#include <list>
using namespace std;

#define BRACKET_MIN_LEN  9 //such as <html> <body> <span> ..., will ignore

struct bracket_t
{
	int start;
	int end;
};
void del_brackets(list<bracket_t*>& brackets)
{
	if(brackets.size() == 0)
		return;
	list<bracket_t*>::iterator it = brackets.begin();
	for(; it != brackets.end(); it++)
	{
		delete (*it);
	}
	brackets.clear();
	
}

int find_brackets(char* p, int len, 
				  list<bracket_t*>& brackets, int& bracket_maxlen,
				  list<bracket_t*>& notes)
{
	// <html> --> <html>, have '<' and '>'
	bool nocheck_notestart = false;
	char* cur = p;
	char* start = 0,*end = 0;
	char* notestart = 0, *noteend = 0;
	while(cur != 0 && cur-p<len )
	{
		start = strchr(cur, '<'); 
		if( start == 0 )		break;
		if(!nocheck_notestart) 	notestart = strstr(cur,"<!--"); //reduce strstr function 
		if( notestart == 0)		nocheck_notestart = true;	
		if(start < notestart || ( notestart == 0 && start > notestart))
		{
			end = strchr(start, '>');
			if( end != 0)
			{
				bracket_t* bracket = new bracket_t();
				bracket->start = start - p ;  
				bracket->end = end - p+1;
				brackets.push_back(bracket);
				cur = end+1;//sizeof('>');
				bracket_maxlen = (bracket_maxlen >= end - start+1)?bracket_maxlen:end - start+1;
			}
			else
			{
				break;
			}
		}
		else 
		{
			// have '<!--' and '-->'
			noteend = strstr(notestart, "-->");
			if( noteend != 0)
			{
				bracket_t* bracket = new bracket_t();
				bracket->start = notestart - p ;
				bracket->end = noteend - p + 3;
				notes.push_back(bracket);
				cur = noteend+3;//sizeof("-->");;
			}
			else
			{
				break;
			}
		}			
	}
	return 0;
}

bool filter_charset(char* p, int len, page_t* page)
{
	char* charset =  strstr(p, "charset=");
	if( charset != 0)
	{
		charset += 8; //strlen("charset=");
		if( strstr(charset, "utf-8") != 0)
		{
			page->charset_type = eCHARSET_UTF8;
		}
		else if( strstr(charset, "gbk") != 0)
		{
			page->charset_type = eCHARSET_GBK;
		}
		else if( strstr(charset, "gb2312") != 0)
		{
			page->charset_type = eCHARSET_GB2312;
		}
		else
		{
			page->charset_type = eCHARSET_UNKNOWN;
		}
		return true;
	}
	return false;
}

int filter_href(char* p, int len, page_t* page)
{
	int startpos = 0;
	char* start = strstr(p, "href=\""); startpos = 6;
	if( start == 0 ) {start = strstr(p, "href='");startpos = 6;}
	if( start == 0 ) {start = strstr(p, "href=");startpos = 5;}
	if( start == 0) return -1;
	start += startpos;
	char* end = strchr(start+3, '\"') ;
	if( end == 0) end = strchr(start+3, '\'');
	if( end == 0) end = strstr(start+3, "/>");
	if( end == 0) end = strchr(start+3, '>');
	if( end == 0) end = strchr(start+3,' ');  //' ' must check in the end
	if( end == 0)	return -1;
	int href_len = end - start;	
	if( href_len < 0 || href_len >= URL_LEN)
		return -1;
	add_href_to_page(start, href_len, page);
	return 0;
}

int filter_bracketcontent(char* p, int len, page_t* page)
{
	//<meta
	int startpos = 0;
	char* start = strstr(p, "<meta"); startpos = 6;
	if( start == 0) return -1;
	start += startpos;
	char* end = strstr(start+8,"/>"); 
	if( end == 0) end = strstr(start+3, ">");
	if( end == 0)	return -1;
	int bracket_len = end - start;	
	if( bracket_len < 0 || bracket_len >= URL_LEN)
		return -1;
	add_bracketcontent_to_page(start, bracket_len, page);	
	return 0;
}

int filter_bracketcontents(char* p, int len,
					 list<bracket_t*>& brackets, int& bracket_maxlen,
					 page_t* page)
{
	if(brackets.size() == 0)
		return -1;

	bool found_charset = false;
		
	char* content = new char[bracket_maxlen+1]; 
	int content_len = 0;
	bracket_t* bracket = 0;
	list<bracket_t*>::iterator it = brackets.begin();
	for(; it!=brackets.end(); it++)
	{
		bracket = (*it);
		if( bracket->end - bracket-> start < BRACKET_MIN_LEN)
			continue;
			
		memset(content, 0, bracket_maxlen+1);
		content_len = bracket->end-bracket->start;
		memcpy(content, p+bracket->start, bracket->end-bracket->start);
		
		if(!found_charset) 
		{
			found_charset = filter_charset(content,content_len, page);
			continue;
		}
		filter_href(content, content_len, page);
		filter_bracketcontent(content, content_len, page);
	}	
	delete[] content;
		
	return 0;
}

bool is_useless_content(char*& start, char*& end)
{
	if( end == 0 || start == 0 || end <= start +1 )
		return true;
		
	//del \r \n \t ' ' from start	
	char* newstart = start; 
	while( newstart != 0 && newstart <= end && 
		  (*newstart == '\r' || *newstart == '\n' 
		  || *newstart == '\t' || *newstart == ' '))
	{
		newstart++;	
	}
	if( newstart >= end)
		return true;
	start = newstart;
	
	//del \r \n \t ' ' from end	
	char* newend = end;
	while( newend != 0 && newend >= start && 
	      (*newend == '\r' || *newend == '\n' 
	      || *newend == '\t' || *newend == ' '))
	{
		newend--;
	}
	if( newend <= start)
		return true;
	end = newend;
	
	//del "&nbsp;"
	newstart = start;
	while(newstart != 0 && newstart <= end )
	{
		if(strstr(newstart, "&nbsp;") == newstart)
		{
			newstart += 6; //&nbsp;
		}
		else
		{
			break;
		}
	}
	if( newstart >= end)
		return true;
	start = newstart;
	
	return false;
}


int filter_body(char* p , int len,
				list<bracket_t*>& brackets, list<bracket_t*>& notes,
				page_t* page)
{
	vector<int> body_poses; // not pointer
	body_poses.reserve(brackets.size() + notes.size());
	
	list<bracket_t*>::iterator it = brackets.begin();
	for(; it != brackets.end(); it++)
	{
		body_poses.push_back((*it)->start);
		body_poses.push_back((*it)->end);
	}
	
	it = notes.begin();
	for(; it != notes.end(); it++)
	{
		body_poses.push_back((*it)->start);
		body_poses.push_back((*it)->end);
	}
	stable_sort(body_poses.begin(), body_poses.end());
	int start = 0, end = 0;
	char* newstart = 0; char* newend = 0;
	body_t& body = page->body;
	body.p = new char[len+1];
	memset(body.p, 0, len+1);
	for(unsigned int i = 0; i < body_poses.size(); i+=2)
	{
		end = body_poses[i];
		if( start < end && (start +1 != end) )
		{
			newstart = p + start;
			newend = p + end;
			if(!is_useless_content(newstart, newend))
			{
				memcpy(body.p+body.len, newstart, newend-newstart);
				body.len += (newend - newstart );
			//	memcpy(body.p+body.len, " ", 1); //insert space
				strcat(body.p, " ");
				body.len++;
			}
		}
		start = body_poses[i+1];
	}
	return 0;
}

int init_page(page_t* page, url_t* url)
{
	if(page == 0 || url == 0)
		return -1;
		
	page->ref_count = 0;
	pthread_mutex_init(&page->locker_ref_count, NULL);
	page->charset_type = eCHARSET_UTF8; //default
	memset(page->title, 0, sizeof(page->title));
	memset(page->host, 0 , sizeof(page->host));
	init_body(&page->body);
	
	copy_url(url, &page->url);
	copy_allhost(url, page->host, HOST_LEN);
	page->keyword = create_keyword(url);
	return 0;
}

int parse(url_t* url, struct memory_block* m)
{
	if( url == NULL || m == NULL || m->pblock == 0 || m->len == 0)
		return -1;
	
	strlwr((char*)m->pblock); //to lower
	
	page_t * page = new page_t();
	init_page(page,url);
	
	list<bracket_t*> brackets;
	list<bracket_t*> notes;
	int bracket_maxlen = 0;
	find_brackets((char*)m->pblock, m->len, brackets,bracket_maxlen,notes);
	filter_bracketcontents((char*)m->pblock, m->len, brackets,bracket_maxlen,page);
	filter_body((char*)m->pblock, m->len,brackets, notes, page);
	del_brackets(brackets);
	del_brackets(notes);
	
	page->check_time = time(NULL);

	add_page_to_dict_manager(page,&g_dict_manager);
	
	return 0;
}
